﻿using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Net;
using System.IO;
using HtmlAgilityPack;
using System.Xml;
using System.Web;
using System.Text.RegularExpressions;

namespace IFTRUE.DataCollection.BLL.Custom
{
    /// <summary>
    /// 作用：网页数据采集类
    /// 作者：iftrue
    /// 日期：2013-10-12
    /// </summary>
  public  class WebPageCollectionBLL
  {
      #region 京东商城数据分析
      //京东商城GUID
      public static string jd_guid = "73EF2CB4-6A79-44BC-9573-C29D6A171BEA";

      /// <summary>
      /// 京东商品分类数据
      /// </summary>
      /// <returns></returns>
      public static List<Model.commodityClass> JDCommodityClass()
      {
          List<Model.commodityClass> commodityClassList = null;
          HtmlDocument doc = GetWebPageDocument("http://www.jd.com/allSort.aspx?dt="+DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss"), Encoding.GetEncoding("gb2312"), "//div[@id='allsort']");
          if (doc != null)
          {
              commodityClassList = new List<Model.commodityClass>();
              HtmlNodeCollection nodeList = doc.DocumentNode.SelectNodes("//div[@class='mt']");
              if (nodeList != null && nodeList.Count > 0)
              {
                  Model.commodityClass commodityClassModel = null;
                  HtmlNodeCollection nodeList2 = null;
                  BLL.commodityClass commodityClassBLL = new commodityClass();
                  foreach (HtmlNode hn in nodeList)
                  {
                      //解析第一分类
                      string strGuid = Guid.NewGuid().ToString();
                      commodityClassModel = new Model.commodityClass();
                      commodityClassModel.guid = strGuid;
                      commodityClassModel.createDate = DateTime.Now;
                      commodityClassModel.level = 1;
                      commodityClassModel.name = hn.InnerText.Replace("\r","").Replace("\n","").Trim();
                      commodityClassModel.parentGuid = jd_guid;
                      commodityClassModel.updateDate = commodityClassModel.createDate;
                      commodityClassList.Add(commodityClassModel);
                      //解析第二分类
                      nodeList2 = hn.ParentNode.SelectSingleNode("div[@class='mc']").SelectNodes("dl/dd/em");
                      if (nodeList != null && nodeList2.Count > 0)
                      {
                          foreach (HtmlNode hn2 in nodeList2)
                          {
                              commodityClassModel = new Model.commodityClass();
                              commodityClassModel.guid = Guid.NewGuid().ToString();
                              commodityClassModel.createDate = DateTime.Now;
                              commodityClassModel.level = 2;
                              commodityClassModel.name = hn2.InnerText.Replace("\r", "").Replace("\n", "").Trim();
                              commodityClassModel.parentGuid = strGuid;
                              commodityClassModel.url = hn2.SelectSingleNode("//a").Attributes["href"].Value;
                              commodityClassModel.updateDate = commodityClassModel.createDate;
                              commodityClassList.Add(commodityClassModel);
                          }

                      }
                  }
              }
          }

          return commodityClassList;
      }
      
      /// <summary>
      /// 京东商品信息
      /// </summary>
      /// <param name="code">商品编号</param>
      /// <returns></returns>
      public static Model.commodity JSCommodity(long code, Encoding encode)
      {
          string strUrl = "http://item.jd.com/"+code+".html";
          return JSCommodity(strUrl, encode);
      }

      /// <summary>
      /// 京东商品信息
      /// </summary>
      /// <param name="url">商品网址</param>
      /// <returns></returns>
      public static Model.commodity JSCommodity(string url, Encoding encode)
      {
          Model.commodity model = null;
          try
          {
              HtmlDocument doc = GetWebPageDocument(url, encode);
              if (doc == null||doc.DocumentNode.OuterHtml.Trim().Length==0)
                  return null;
              model = new Model.commodity();
              model.mall = "京东商城";
              model.url = url;

              Model.Custom.Commodity_JD jdModel = new Model.Custom.Commodity_JD();

              HtmlNode node = doc.DocumentNode.SelectSingleNode("//div[@class='breadcrumb']");
              if (node != null)
              {
                  model.commodityClass = HttpUtility.HtmlDecode(node.InnerText.Trim());
              }
              node = doc.DocumentNode.SelectSingleNode("//div[@id='name']");
              if (node != null)
              {
                  model.title = node.InnerText.Trim().Replace("\r", "").Replace("\n", "").Replace("\t", "").Trim();
              }
              node = doc.DocumentNode.SelectSingleNode("//div[@id='product-detail-1']").SelectSingleNode("ul/li[1]");
              if (node != null && node.InnerText.Split('：').Length>1)
              {
                  model.name = node.InnerText.Trim().Split('：')[1];
              }
              node = doc.DocumentNode.SelectSingleNode("//div[@id='product-detail-1']").SelectSingleNode("ul/li[2]");
              if (node != null && node.InnerText.Split('：').Length > 1)
              {
                  model.code = node.InnerText.Trim().Split('：')[1];
              }

              node = doc.DocumentNode.SelectSingleNode("//div[@id='product-detail-1']").SelectSingleNode("ul/li[3]");
              if (node != null && node.InnerText.Split('：').Length > 1)
              {
                  model.brands = node.InnerText.Trim().Split('：')[1];
              }
              //价格是动态数据，需要重新获取
              HtmlDocument hdoc = GetWebPageDocument("http://p.3.cn/prices/mgets?type=1&skuIds=J_" + model.code, encode);
              if (hdoc != null)
              {
                  string s = hdoc.DocumentNode.OuterHtml.Replace("\r", "").Replace("\n", "").Trim();
                  jdModel.Price = Newtonsoft.Json.JsonConvert.DeserializeObject<Model.Custom.Commodity_JD_Price>(s.Substring(1, s.Length - 2));
                  model.price = jdModel.Price.P;
              }
              node = doc.DocumentNode.SelectSingleNode("//div[@id='product-detail-1']").SelectSingleNode("ul/li[4]");
              if (node != null && node.InnerText.Split('：').Length > 1)
              {
                  jdModel.Other.AddTime = node.InnerText.Trim().Split('：')[1];
              }
              //副标题是动态数据，需要重新获取
              HtmlDocument hd = GetWebPageDocument("http://jprice.jd.com/adslogan/" + model.code + "-setproductadwords.ad", encode);
              if (hd != null)
              {
                  string s = hd.DocumentNode.OuterHtml.Replace("\r", "").Replace("\n", "").Trim();
                  if (s.IndexOf("\\u") > -1)
                  {
                      s = s.Substring(s.IndexOf("\\u"), s.LastIndexOf("\\u") - s.IndexOf("\\u"));
                      s = IFTRUE.DataCollection.Common.StringPlus.UnicodeToChinese(s);
                      jdModel.Other.Subtitle = s;
                  }
              }
              model.info = Newtonsoft.Json.JsonConvert.SerializeObject(jdModel);
              node = doc.DocumentNode.SelectSingleNode("//div[@class='itemover-title']");
              if (node != null && node.InnerText.IndexOf("下柜") > -1)
              {
                  model.status = "下柜";
              }
              else
              {
                      model.status = "在售";
              }
              model.createDate = DateTime.Now;
              model.updateDate = DateTime.Now;
          }
          catch (Exception ex)
          {
              if (model != null && model.code.Length > 0)
              {
                  model.status = "异常";
                  model.createDate = DateTime.Now;
                  model.updateDate = DateTime.Now;
              }
              else
                  return null;
          }
      
          return model;
      }

      #endregion

      #region 商城数据

      /// <summary>
      /// 商城列表
      /// </summary>
      /// <returns></returns>
      public static List<Model.mall> MallList()
      {
          return new BLL.mall().GetModelList("");
      }

      #endregion

      #region 网页数据获取

      /// <summary>
      /// 获取网页内容
      /// </summary>
      /// <param name="url">网页地址</param>
      /// <param name="encode">编码格式</param>
      public static HtmlDocument GetWebPageDocument(string url, Encoding encode)
      {
          //路径后添加时间戳，防止读取缓存数据
          if (url.IndexOf("?") > 0)
          {
              url += "&_=" + Common.TimeParser.GetTimestamp();
          }
          else
          {
              url += "?_=" + Common.TimeParser.GetTimestamp();
          }
          WebClient client = new WebClient();
          HtmlDocument htmlDoc = null;
          using (MemoryStream ms = new MemoryStream(client.DownloadData(url)))
          {
              htmlDoc = new HtmlDocument();
              htmlDoc.Load(ms, encode);
          }
          client = null;
          return htmlDoc;
      }

      /// <summary>
      /// 获取网页内容
      /// </summary>
      /// <param name="url">网页地址</param>
      /// <param name="encode">编码格式</param>
      /// <param name="xpath">查询表达式</param>
      public static HtmlDocument GetWebPageDocument(string url, Encoding encode, string xpath)
      {
          if(string.IsNullOrEmpty(xpath))
            return  GetWebPageDocument(url, encode);
          HtmlDocument htmlDoc = GetWebPageDocument(url, encode);
          HtmlDocument doc = new HtmlDocument();
          try
          {
              doc.LoadHtml(htmlDoc.DocumentNode.SelectSingleNode(xpath).InnerHtml);
          }
          catch (Exception ex)
          { 
              return null;
          }
          htmlDoc = null;
          return doc;
      }

      /// <summary>
      /// 获取网页HTML
      /// </summary>
      /// <param name="url">网页地址</param>
      /// <param name="encode">编码格式</param>
      public static string GetWebPageHtml(string url, Encoding encode)
      {
          string strHtml = string.Empty;
          WebRequest wrt = WebRequest.Create(url);
          WebResponse wrse = wrt.GetResponse();
          using (Stream stream = wrse.GetResponseStream())
          {
              using (StreamReader sr = new StreamReader(stream,encode))
              {
                  strHtml = sr.ReadToEnd();
              }
          }
          return strHtml.ToString();
      }
      #endregion
  }
}
